# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
import scrapy

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter

import random,base64
from gbifSpider.settings import USER_AGENT_LIST, PROXY_LIST

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from scrapy.http import HtmlResponse
import re

class RandomUserAgent(object):
    def process_request(self, request, spider):
        ua = random.choice(USER_AGENT_LIST)
        request.headers['User-Agent'] = ua


class RedirectMiddleware(object):
    def process_response(self, request, response, spider):
        # 当状态码为301或302时，重新发起请求
        if response.status in [301, 302]:
            return scrapy.Request(url=response.url, dont_filter=True, callback=request.callback)
        # 其他状态码直接返回响应
        return response

class RandomProxy(object):
    def process_request(self, request, spider):
        proxy = random.choice(PROXY_LIST)
        if 'user_password' in proxy:
            b64_up = base64.b64decode(proxy['user_password'])
            request.headers['Proxy-Authorization'] = 'Basic '+ b64_up.decode()
            request.meta['proxy'] = proxy['ip_port']
        else:
            request.meta['proxy'] = proxy['ip_port']

s_reg = '/www.qcc.com|keyword1|keyword1'
class SeleniumMiddleware(object):
    def process_request(self, request, spider):
        url = request.url
        if re.search(s_reg,url):
            options = Options()
            options.add_argument(
                'User-Agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"')
            options.add_argument('--headless')
            options.add_argument('--disable-gpu')
            options.add_argument('--no-sandbox')
            options.add_experimental_option('excludeSwitches', ['enable-automation'])
            options.add_argument('--disable-dev-shm-usage')
            options.add_argument('--window-size=1920,1080')
            driver = webdriver.Chrome(
                executable_path=r'C:\Users\Administrator\AppData\Local\Google\Chrome\Application\chromedriver.exe',
                options=options)
            driver.get(url)
            WebDriverWait(driver, 30).until(lambda d: d.execute_script(
                'return ["complete", "interactive"].indexOf(document.readyState) != -1'))
            data = driver.page_source
            driver.quit()
            res = HtmlResponse(url=url,body=data,encoding='utf-8',request=request)
            return res